# -*- coding: utf8 -*-

import os
import sys
reload(sys) 
sys.setdefaultencoding('utf8')
import caller
import webkit
import webpage
import webelement
import pymongo
import threading
import thread
import time
import datetime

gconfig = caller.GlobalConfig()
gconfig.init("10.58.222.103:2181")

class ParseProduct(threading.Thread):
    def __init__(self, url):
        threading.Thread.__init__(self)
        self.url = url
        self.started = False
        self.state = "init"
    
    def run(self):
        self.started = True
        self.stoped = False
        self.state = "start"
        print "Start " + self.url
        wk = webkit.WebKit("test")
        self.state = "getpage"
        page = None
        while page is None:
            page = wk.getWebPage(self.url, 100, 60000)
            
        self.state = "parse"
        self.parse(page)

        self.state = "stop"
        self.stoped = True

        self.state = "destroy"
        page.destroy()
        wk.release()
        self.state = "die"

    def parse(self, page):

        url = page.getUrl()
        title = page.getTitle()
        product_name = ""
        product_price = ""
        product_detail = ""
        product_detail_table = ""

        client = pymongo.MongoClient("127.0.0.1", 27017)
        db = client.jd_info
        
        self.state = "check"
        if url == "http://www.jd.com/":
            print "Get a index page!"
            now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            db.jd_collection_20140728_error.insert({"url":self.url,"error":"index page", "date":now});
            page.destroy()
            return

        if url == "http://www.jd.com/error.html":
            print "Get a error page!"
            now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            db.jd_collection_20140728_error.insert({"url":self.url,"error":"error page", "date":now});
            page.destroy()
            return

        if url == "about:blank":
            print "Get a blank page"
            now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            db.jd_collection_20140728_error.insert({"url":self.url,"error":"blank page", "date":now});
            page.destroy()
            return

        if url == "":
            print "Get a null page"
            now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            db.jd_collection_20140728_error.insert({"url":self.url,"error":"null page", "date":now});
            return

        self.state = "name"
        if page.waitForCssSelector("#name > h1:nth-child(1)", 1000):
            element = page.getElementByCssSelector("#name > h1:nth-child(1)")
            if element.waitForInnerXml(1000):
                product_name = element.getInnerXml()
                #print product_name
            else:
                now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                db.jd_collection_20140728_error.insert({"url":url, "titel":title, "error":"name content", "source":page.getPageSource(), "date":now});
                print "没有拿到商品名称内容"
                return
        else:
            now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            db.jd_collection_20140728_error.insert({"url":url, "titel":title, "error":"name", "source":page.getPageSource(), "date":now});
            print "没有拿到商品名称"
            return

        self.state = "price"
        if page.waitForCssSelector("#jd-price", 1000):
            element = page.getElementByCssSelector("#jd-price")
            if element.waitForInnerXml(1000):
                product_price = element.getInnerXml()
                #print product_price
            else:
                now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                db.jd_collection_20140728_error.insert({"url":url, "titel":title, "error":"price content", "source":page.getPageSource(), "date":now});
                print "没有拿到商品价格内容"
                return
        else:
            now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            db.jd_collection_20140728_error.insert({"url":url, "titel":title, "error":"price", "source":page.getPageSource(), "date":now});
            print "没有拿到商品价格"
            return

        self.state = "list"
        if page.waitForCssSelector(".detail-list", 1000):
            element = page.getElementByCssSelector(".detail-list")
            if element.waitForInnerXml(1000):
                product_detail = element.getInnerXml()
                #print product_detail
            else:
                print "没有拿到商品信息内容"
        else:
            print "没有拿到商品信息"

        self.state = "table"
        if page.waitForCssSelector(".Ptable", 10):
            element = page.getElementByCssSelector(".Ptable")
            if element.waitForInnerXml(10):
                product_detail_table = element.getInnerXml()
                #print product_detail_table
            else:
                print "没有拿到规格参数内容"
        elif page.waitForCssSelector("#product-detail-2 > div:nth-child(1)", 10):
            element = page.getElementByCssSelector("#product-detail-2 > div:nth-child(1)")
            if element.waitForInnerXml(10):
                product_detail_table = element.getInnerXml()
                #print product_detail_table
            else:
                print "没有拿到规格参数内容"
        else:
            print "没有拿到规格参数"

        self.state = "mongo"
        now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        obj = {"url":url, "product_price":product_price, "product_name":product_name, "product_detail":product_detail, "product_detail_table":product_detail_table, "date":now}
        print "Save " + url + " | " + product_name
        self.state = "mongo insert"
        db.jd_collection_20140728.insert(obj)
        self.state = "parse end"


def getUrls(host, port):
    urls = []
    client = pymongo.MongoClient(host, port);
    db = client.jd_info
    urlinfos = db.jd_plink_20140723.find()
    for info in urlinfos:
        urls.append(info['url'])
    urls.reverse()
    return urls

parsers = []

def output_info(no, arg):
    while True:
        raw_input()
        for p in parsers:
            print p.url + " " + p.state

def main():
    global parsers

    urls = getUrls("127.0.0.1", 27017)
    print "Get " + str(len(urls)) + " urls."

    for i in range(100):
        parsers.append(ParseProduct(urls.pop()))

    thread.start_new_thread(output_info, (0,0))

    while(len(parsers)):
        for p in parsers:
            if not p.started:
                p.start()
            elif p.stoped:
                parsers.remove(p)
        
        c = 100 - len(parsers)
        if c > 0:
            print "Start: " + str(len(parsers)) + " wait: " + str(len(urls))
            if len(urls) > c:
                for i in range(c):
                    parsers.append(ParseProduct(urls.pop()))
            else:
                for i in range(len(urls)):
                    parsers.append(ParseProduct(urls.pop()))
                    
        time.sleep(0.1)

main()


